package searchme.util;

import java.io.*;

import java.net.*;
import java.util.Arrays;
import java.util.Scanner;
import java.util.HashMap;
import java.util.Vector;
import java.util.Iterator;
import java.util.List;

import java.lang.Math;

import javax.xml.parsers.*;

import org.w3c.dom.*;
import org.xml.sax.*;

public class SearchMe {

	private Vector<String> query = new Vector<String>();
	private String bestTerm = "";
	private String secondBestTerm = "";
	private double precision = 0.0; // desired precision
	private double round = 0; // number of iterations requiring user feedback
	private static String appId;
	private final static Stopword stopwordList = new Stopword();
	// weight for relevant document abstract
	private final static double alpha = 0.25;
	// weight for relevant document title
	private final static double beta = 0.5;
	// weight for irrelevant document
	private final static double gamma = 0.15;
	// window size (used for proximity)
	private final static int window = 2;
	private double currentPrecision = 0.0;
	// number of documents Yahoo returns
	int N = 10;
	SearchResult[] resultset = null;
	private HashMap<String, Double> invertedFile = new HashMap<String, Double>();

	/**
	 * rearrange the query words
	 * @param query
	 * @return
	 */
	String rearrangeQuery(Vector<String> query) {
		String newQuery = "";
		int[] value = new int[query.size()];
		String[] words = new String[query.size()];
		//calculate average position value
		for (int i = 0; i < query.size(); i++) {
			words[i] = (String) query.elementAt(i);
			int num = 0;
			for (SearchResult search : resultset) {
				if (search.isRelevant && search.hasWord(words[i])) {
					num++;
					value[i] = search.abstractTxt.toLowerCase().indexOf(
							words[i]);
				}
			}
			if (num != 0)
				value[i] = value[i] / num;
			System.out.println(value[i]);
		}
		//rank the keywords based on the position value
		for (int i = 0; i < query.size() - 1; i++) {
			for (int j = 0; j < query.size() - 1 - i; j++) {
				if (value[j] > value[j + 1]) {
					int tempInt = value[j];
					value[j] = value[j + 1];
					value[j + 1] = tempInt;
					String tempString = words[j];
					words[j] = words[j + 1];
					words[j + 1] = tempString;
				}
			}
		}
		newQuery = words[0];
		for (int i = 1; i < query.size(); i++) {
			newQuery = newQuery + "%20" + words[i];
		}
		return newQuery;
	}

	/**
	 * This function is responsible for real search request invocation and
	 * response handling
	 * 
	 * @param keyword
	 *            stands for the search keyword
	 */
	public void search() {
		try {
			// replace all spaces with "%20" to get url friendly query
			String urlQuery = "";
			urlQuery = query.elementAt(0);
			for (int i = 1; i < query.size(); i++) {
				urlQuery = urlQuery + "%20" + query.elementAt(i);
			}
			URL url = new URL("http://boss.yahooapis.com/ysearch/web/v1/'"
					+ urlQuery + "'?appid=" + appId + "&format=xml");

			URLConnection con = url.openConnection();
			InputStream inStream = con.getInputStream();
			Scanner in = new Scanner(inStream);
			StringBuffer temp = new StringBuffer();
			while (in.hasNextLine()) {
				temp.append(in.nextLine());
			}
			String res = temp.toString();
			parseSearchResult(res);

		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

	private void printInvertedFile() {
		System.out.println("Retriving all keys from the inverted file");
		Iterator<String> iterator = invertedFile.keySet().iterator();
		while (iterator.hasNext()) {
			String word = (String) (iterator.next());
			System.out.println(word + " - " + invertedFile.get(word));
		}
	}

	/**
	 * buildInvertedFile - populates the inverted index with additional terms
	 * extracted from the text
	 * 
	 * @param text
	 *            the text content (could be title or abstract of a document)
	 * @param type
	 *            the text type (0 = title, 1 = abstract of a document)
	 * @param relevant
	 *            text relevance (0 = irrelevant, 1 = relevant)
	 */
	private void buildInvertedFile(String text, int type, int relevant) {
		double multiplier = (relevant == 1) ? ((type == 1) ? alpha : beta)
				: -gamma;
		Double value;
		text = text.toLowerCase().replaceAll("[^{a-z| }]", "");
		text = text.replaceAll("[ ]+", " ");
		String words[] = text.split(" ");
		for (int i = 0; i < words.length; i++) {
			// first check whether the word is stopword, if it is stopword it
			// would not be added to list
			if (!stopwordList.isStopword(words[i]))
				if (invertedFile.containsKey(new String(words[i]))) {
					value = (Double) invertedFile.get(new String(words[i]));
					value = value + multiplier;
					invertedFile.put(new String(words[i]), value);
				} else {
					invertedFile.put(new String(words[i]), new Double("1"));
				}
		}
	}

	private Boolean isInQuery(String s) {
		for (int i = 0; i < query.size(); i++) {
			if (query.elementAt(i).equals(s))
				return true;
		}
		return false;
	}

	private void getTheTopTwoTerms() {
		Double bestWeight = new Double("0");
		bestTerm = "";
		Double secondBestWeight = new Double("0");
		secondBestTerm = "";
		String term;
		Double termWeight;
		Iterator<String> iterator = invertedFile.keySet().iterator();
		while (iterator.hasNext()) {
			term = (String) (iterator.next());
			termWeight = invertedFile.get(term);
			if (!isInQuery(term)) {
				if (termWeight >= bestWeight) {
					secondBestWeight = bestWeight;
					secondBestTerm = bestTerm;
					bestWeight = termWeight;
					bestTerm = term;
				} else {
					if (termWeight >= secondBestWeight) {
						secondBestWeight = termWeight;
						secondBestTerm = term;
					}
				}
			}
		}
	}

	private void modifyQuery() {
		int i, k, p, index, arrIndex;
		// the following two terms need to be inserted in the query in the best place
		String[] newTerms = new String[2];
		newTerms[0] = bestTerm;
		newTerms[1] = secondBestTerm;
		String text;
		// we use proximity table
		// proximity[i][j] is the minimum distance between new term i and keyword j (in the query)
		// negative proximity means new term i is to the left of keyword j
		// positive proximity means new term i is to the right of keyword j
		int[][] proximity = new int[2][query.size()];
		for (i = 0; i < query.size(); i++) {
			proximity[0][i] = Integer.MAX_VALUE;
			proximity[1][i] = Integer.MAX_VALUE;
		}
		if (resultset != null) {
			for (SearchResult sr : resultset) {
				for (i = 0; i < 2; i++) {
					if (sr.isRelevant) {
						text = sr.abstractTxt.toLowerCase() + " " + sr.title.toLowerCase();
						text = text.toLowerCase().replaceAll("[^{a-z| }]", "");
						text = text.replaceAll("[ ]+", " ");
						String words[] = text.split(" ");

						for (k = 0; k < words.length; k++) {
							if (words[k].equals(newTerms[i])) {
								// check if there is a keyword in proximity p to the right of the new term i
								for (p = 1; p <= window; p++) {
									arrIndex = k+p >= words.length ? words.length-1 : k+p; 
									index = query.indexOf(words[arrIndex]);
									if (index != -1) {
										if (Math.abs(proximity[i][index]) > p) {
											proximity[i][index] = -p;
										}
									}
								}
								// check if there is a keyword in proximity p to the left of the new term i
								for (p = 1; p <= window; p++) {
									arrIndex = k-p < 0 ? 0 : k-p;
									index = query.indexOf(words[arrIndex]);
									if (index != -1) {
										if (Math.abs(proximity[i][index]) > p) {
											proximity[i][index] = p;
										}
									}
								}
							}
						}
					}
				}
			}
		}
		
		int[] minProximity = new int[2]; // min distance from new term i to any keyword k in the query
		int[] minProximityIndex = new int[2]; // the index of the closest keyword k in the query
		String[] minProximityKeyword = new String[2]; // the the closest keyword k in the query
		minProximity[0] = minProximity[1] = Integer.MAX_VALUE;
		minProximityIndex[0] = minProximityIndex[1] = query.size()-1;
		for (i = 0; i < 2; i++) {
			for (k = 0; k < query.size(); k++) { // run over all keywords in the query in order to find the closest to new term i
				if (Math.abs(minProximity[i]) > Math.abs(proximity[i][k])) {
					minProximity[i] = proximity[i][k];
					minProximityIndex[i] = k;
					minProximityKeyword[i] = query.elementAt(k);
				}
			}
		}

		if (minProximity[0] * minProximity[1] > 0) { // order of insertions matters
			if (Math.abs(minProximity[0]) > Math.abs(minProximity[1])) { 
				insertNewTermInQuery(minProximity[0], newTerms[0], minProximityKeyword[0]);
				insertNewTermInQuery(minProximity[1], newTerms[1], minProximityKeyword[1]);
			} else {
				insertNewTermInQuery(minProximity[1], newTerms[1], minProximityKeyword[1]);
				insertNewTermInQuery(minProximity[0], newTerms[0], minProximityKeyword[0]);
			}
		} else { // order of insertions does not matters
			insertNewTermInQuery(minProximity[0], newTerms[0], minProximityKeyword[0]);
			insertNewTermInQuery(minProximity[1], newTerms[1], minProximityKeyword[1]);
		}	
	}
	
	private void insertNewTermInQuery(int proximity, String term, String keyword) {
		int index = query.indexOf(keyword);
		if (proximity > 0) index++; // if we need to insert to the right of the keyword
		if (!term.equals(""))
			query.insertElementAt(term, index > 0 ? index : 0);
	}

	private Boolean isThisRelevant() {
		System.out.print("Is this document relevant to your search? (y/n)");
		BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
		try {
			String a;
			while (true) {
				a = br.readLine();
				if (a.equals("y") || a.equals("Y")) {
					return true;
				}
				if (a.equals("n") || a.equals("N")) {
					return false;
				}
				System.out
						.print("Is this document relevant to your search? (y/n)");
			}
		} catch (IOException e) {
			e.printStackTrace();
			return false;
		}
	}

	/**
	 * use DOM to parse XML data and retrieve results from response
	 * 
	 * @param response
	 *            returned XML data
	 */
	private void parseSearchResult(String response) {
		resultset = new SearchResult[N];
		int R = 0; // number of relevant documents

		DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
		DocumentBuilder builder;
		try {
			builder = factory.newDocumentBuilder();
			Document doc = builder.parse(new InputSource(new StringReader(
					response)));
			NodeList nodeL = doc.getElementsByTagName("result");
			Node node;
			N = nodeL.getLength();
			for (int i = 0; i < nodeL.getLength(); i++) {
				node = nodeL.item(i);
				resultset[i] = new SearchResult(node);
				resultset[i].isRelevant = isThisRelevant();
				if (resultset[i].isRelevant) {
					R++;
					buildInvertedFile(resultset[i].title, 0, 1);
					buildInvertedFile(resultset[i].abstractTxt, 1, 1);
				} else {
					buildInvertedFile(resultset[i].abstractTxt, 0, 0);
					buildInvertedFile(resultset[i].abstractTxt, 1, 0);
				}
			}
			currentPrecision = (double) R / N;
		} catch (ParserConfigurationException e) {
			e.printStackTrace();
		} catch (SAXException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public static void main(String args[]) throws Exception {
		int maxIterations = 30;
		SearchMe search = new SearchMe();
		if (args.length != 3) {
			System.out.println("Usage: SearchMe <query-surrounded-in-quotes> <precision> <yahoo appId>");
			System.exit(1);
		}
		String[] keywords = args[0].split(" ");
		List<String> c = Arrays.asList(keywords);
		search.query.addAll(c);

		search.precision = Double.parseDouble(args[1]);
		search.appId = args[2];

		while (search.precision > search.currentPrecision
				&& search.round < maxIterations) {
			search.modifyQuery();
			System.out.println("the extended query is "
					+ search.query);
			search.search();
			search.getTheTopTwoTerms();
			search.round++;
			System.out.println("current round is " + search.round);
			System.out.println("current precision is "
					+ search.currentPrecision);
			System.out.println("best term is " + search.bestTerm);
			System.out.println("second best term is "
					+ search.secondBestTerm);
			if (search.currentPrecision == 0) {
				System.out.println("No relevant pages");
				break;
			}
		}

		System.out
				.println("The query used at the final round is:");
		for (int k = 0; k < search.query.size(); k++) {
			System.out.println(search.query.elementAt(k));
		}
	}
}
